package com.shujia.spark

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object Demo25Bro {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf()
      .setMaster("local")
      .setAppName("Demo23Cache")

    val sc = new SparkContext(conf)

    val studentsRDD: RDD[String] = sc.textFile("data/students.txt")


    /**
      * 取出集合中的所有学生的信息
      *
      */
    val ids = List("1500100006", "1500100011", "1500100016", "1500100041")

    /*    val filterRDD: RDD[String] = studentsRDD.filter(student => {
          val id: String = student.split(",")(0)

          ids.contains(id)
        })

        filterRDD.foreach(println)*/

    /**
      * 使用广播变量
      *
      */

    //1、将Driver端的一个变量广播出去
    val broIds: Broadcast[List[String]] = sc.broadcast(ids)

    val filterRDD: RDD[String] = studentsRDD.filter(student => {
      val id: String = student.split(",")(0)

      //2、在task中获取广播变量
      val value: List[String] = broIds.value

      value.contains(id)
    })

    filterRDD.foreach(println)

  }

}
